# coding=utf-8

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import *
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from time import sleep
import re
import codecs
import sys
import os
import platform
# import math
# import requests
# import json
from disease_model import *
from diseaseimg_model import *
from mysql import *


class Disease:
    '''
    https://www.google.com.hk/imghp
    Google-丁香疾病图片数据库
    @author spring
    @date 2022.02.24
    '''

    # 起始页
    #__start_page = 'https://www.google.com.hk/imghp'
    __start_page = 'https://www.google.com.hk/search?q=irregular+menstruation+disease+introduction&newwindow=1&tbm=isch&source=hp&biw=1087&bih=920&ei=UCEXYuPgGoqVr7wPzpSB-AQ&iflsig=AHkkrS4AAAAAYhcvYOKWxsuEDByY4DjgDOfE4pDJJ4Rv&ved=0ahUKEwjjyNaA15f2AhWKyosBHU5KAE8Q4dUDCAY&oq=irregular+menstruation+disease+introduction&gs_lcp=CgNpbWcQDFAAWDBgT2gAcAB4AIABAIgBAJIBAJgBAKABAaoBC2d3cy13aXotaW1n&sclient=img'
    #__start_page = 'https://www.google.com.hk/search?q=Spontaneous+Abortion+disease+introduction&tbm=isch&ved=2ahUKEwiM9rqCt6b2AhWfADQIHZ54AKUQ2-cCegQIABAA&oq=Spontaneous+Abortion+disease+introduction&gs_lcp=CgNpbWcQA1C2BVi2BWDjC2gAcAB4AIABkwOIAeYEkgEHMi0xLjAuMZgBAKABAaoBC2d3cy13aXotaW1nwAEB&sclient=img&ei=Ct0eYozJE5-B0PEPnvGBqAo&bih=920&biw=1087'

    # 浏览器对象
    __browser = None

    # 当前id
    __current_id = 1

    __max_id = 1157

    # 当前条数，用来处理点击到第几个了
    __current_index = 0

    # 进程名称
    __title = 'Google-丁香疾病图片数据库'

    # 抓取状态
    __status = {'RUN': '抓取中', 'STOP': '停止', 'FINISH': '完成'}

    model = None

    # 记录采集位置，格式：采集当前页数的第几条
    log_file = sys.path[0] + '/log_disease.data'

    env_file = sys.path[0] + '/.env'

    def init_chrome(self):
        '''
        初始化浏览器
        '''

        # options
        chrome_options = webdriver.ChromeOptions()


        #chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument('--no-sandbox')

        proxy = None

        if platform.system() == 'Windows':
            chrome_options.binary_location = "C:\Program Files\Google\Chrome\Application\chrome.exe"
        else:
            chrome_options.add_argument('--headless')
            chrome_options.add_argument('--disable-gpu')
            chrome_options.add_argument('--no-sandbox')
            # proxy = self.get_proxy_ip()

        if proxy:
            print('获取代理IP：', proxy)
            # 设置代理
            chrome_options.add_argument('--proxy-server=%s' % proxy)

        self.__browser = webdriver.Chrome(options=chrome_options)
        self.__browser.set_window_size(1100, 1050)

    def start(self):

        # 读取日志数据
        self.get_collect_log()

        try:
            self.init_chrome()
            self.__browser.get(self.__start_page)
        except WebDriverException as err:
            print('初始化出错')
            os._exit(0)

        #self.__browser.implicitly_wait(3)
        # 直接运行循环抓取过程
        self.collect_list()

        # self.__browser.execute_script("var q=document.documentElement.scrollTop=10000")
        # sleep(1)
        # if self.is_element_present(By.CLASS_NAME, 'r0zKGf'):
        #     # 修改当前疾病的更多开始状态
        #     self.__browser.find_element_by_class_name('r0zKGf').click()

        # sleep(3)
        # img_list = self.__browser.find_elements(By.CLASS_NAME, 'Q4LuWd')
        #
        # # 每个进行点击
        # list_len = len(img_list)
        # print('len:', list_len)
        # sys.exit();

    def collect_list(self):

        print('采集2id：', self.__current_id)
        self.disease = Disease_Model()
        # 获取一条进行处理
        vo = self.disease.get(str(self.__current_id))
        #print(vo)
        if vo:
            # 获取当前执行的id，拿到关键字，拼接以后放入搜索框

            # 获取当前脚本的用户信息
            self.__current_id = vo['id']
            self.info = vo
            # # 赋值输入框
            keywords = self.info['disease_name_en'] + " disease introduction"
            input_q = self.__browser.find_element_by_class_name('og3lId')
            login_form = self.__browser.find_element_by_class_name('wQnou')
            # 登录
            input_q.clear()
            input_q.send_keys(keywords)
            login_form.submit()
            sleep(3)
            img_list_nums = 51;
            img_list = self.__browser.find_elements(By.CLASS_NAME, 'Q4LuWd')

            # 每个进行点击
            list_len = len(img_list)
            # print('len:',list_len)
            # sys.exit();
            if (list_len < 51):
               img_list_nums = self.scroll()

            print("total_num：",img_list_nums)

            #sleep(2)
            #nums = range(50)
            sub_str = "PDF"
            # 目标是50个图片，通过点击来获取，如果不够，就进行滚动加载
            index = 51
            img_index = self.__current_index + 1
            for i in range(400): # 如果400个都还不够，那就不管了
                if i < self.__current_index:
                    continue
                # 如果没有了就跳出去
                if i >= len(self.__browser.find_elements(By.CLASS_NAME, 'wXeWr')):
                    break
                #sleep(1)
                # 点击图片，获取图片的标题和图片地址
                if self.__browser.find_elements(By.CLASS_NAME, 'wXeWr')[i]:
                    img_a = self.__browser.find_elements(By.CLASS_NAME, 'wXeWr')[i]
                    # img_a.click()
                    self.__browser.execute_script(
                        "document.getElementsByClassName('wXeWr')[" + str(i) + "].click()")

                    if self.is_element_present(By.CLASS_NAME, 'n3VNCb'):
                        if len(self.__browser.find_elements(By.CLASS_NAME, 'n3VNCb')) < 2:
                            sleep(1)
                        img_info = self.__browser.find_elements(By.CLASS_NAME, 'n3VNCb')[1]
                        img_title = img_info.get_attribute("alt")
                        # 获取被点击图片的a标签href信息
                        img_url = img_a.get_attribute("href")
                        # 判断alt是否存在PDF
                        if img_title.find(sub_str) == -1:
                            if not(self.obj_is_element_present(By.TAG_NAME, 'span')):
                                did = self.info['id']
                                name = self.info['disease_name_en']
                                base = img_info.get_attribute("src") #点击大图获取到的连接，用来给本地图片进行保存
                                # 入库
                                self.model = Diseaseimg_Model(
                                    did=did,
                                    name=name,
                                    title=img_title,
                                    url=img_url,
                                    base=base,
                                    index=img_index
                                )
                                self.model.save()
                                self.__current_index = int(self.__current_index) + 1
                                self.save_collect_log()
                                img_index = img_index + 1
                    else:
                        img_list_nums = self.scroll()
                else:
                	self.scroll()
                	sleep(2)
                i = i + 1 # 无论是否够，图片都要向下走
                if img_index >= index:
                    break
                # 如果
                if i >= img_list_nums:
                    img_list_nums = self.scroll()

            # 修改当前疾病状态，并开始下一个
            self.disease.update(str(self.__current_id), str(2))


        # if int(self.__current_id) > int(self.__max_id):
        #     self.__current_id = self.__current_id - 1
        #     self.__current_index = 0
        #     # 保存日志
        #     self.save_collect_log()
        #     # 继续采集下一页
        #     self.collect_list()
        # else:
        print('没有下一页，退出')

        self.__current_id = 0
        self.__current_index = 0
        # 保存日志
        self.save_collect_log()
        # 退出
        self.quit_process()

    # 判断元素是否存在
    def is_element_present(self, how, what):
        try:
            self.__browser.find_element(by=how, value=what)
        except NoSuchElementException:
            return False
        return True

    def obj_is_element_present(self, how, what):
        try:
            self.__browser.find_elements(By.CLASS_NAME, 'eYbsle')[1].find_element(by=how, value=what)
        except NoSuchElementException:
            return False
        return True

    # 向下滚动，遇到更多点击更多
    def scroll(self):
        self.__browser.execute_script("var q=document.documentElement.scrollTop=10000")
        self.__browser.execute_script("var q=document.documentElement.scrollTop=10000")
        self.__browser.execute_script("var q=document.documentElement.scrollTop=10000")
        if self.is_element_present(By.CLASS_NAME, 'r0zKGf'):
            # 修改当前疾病的更多开始状态
            self.__browser.find_element_by_class_name('r0zKGf').click()
        img_list = self.__browser.find_elements(By.CLASS_NAME, 'Q4LuWd')
        print("scroll:",len(img_list))
        # 每个进行点击
        list_len = len(img_list)
        return list_len

    def save_collect_log(self):
        '''
        保存抓取位置
        '''
        with codecs.open(self.log_file, 'w', encoding='utf-8') as file:
            file_content = str(self.__current_id) + '-' + str(self.__current_index)
            file.write(file_content)
            file.close()

    def get_collect_log(self):
        '''
        获取抓取位置
        '''
        self.__current_id = 1

        if os.path.exists(self.log_file):
            with codecs.open(self.log_file, 'r', encoding='utf-8') as file:
                file_content = file.read(100)
                file.close()
                if len(file_content) > 0:
                    contents = file_content.split("-")
                    self.__current_id = int(contents[0])
                    self.__current_index = int(contents[1])
                    print('读取文件日志完成: 当前页：%s，当前条数：%s' % (self.__current_id, self.__current_index))
                else:
                    raise '日志格式不正确'
        else:
            print('日志文件不存在...')

    def quit_process(self):
        self.__browser.quit()
        sys.exit()


starter = Disease()
starter.start()
